{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-03T13:09:31.041796Z",
     "start_time": "2025-07-03T13:09:31.026732Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 导入必要的库\n",
    "import torch  # PyTorch库，用于深度学习模型的训练和推理\n",
    "from tqdm import tqdm  # tqdm库，用于显示进度条\n",
    "from modelscope import AutoTokenizer, AutoModel  # modelscope库，用于自动加载预训练模型和分词器\n",
    "from peft import PeftModel, PeftConfig  # peft库，用于进行prompt-tuning高效微调"
   ],
   "id": "c7f709250f90dd9f",
   "outputs": [],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-03T13:09:34.185182Z",
     "start_time": "2025-07-03T13:09:34.169510Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 定义模型和预训练模型的路径\n",
    "#model_dir = \"../../chatglm3-6b\"  # 预训练模型的路径\n",
    "model_dir = \"C:\\\\Users\\\\16014\\\\.cache\\\\modelscope\\\\hub\\\\models\\\\ZhipuAI\\\\chatglm3-6b\"\n",
    "peft_model_id = \"./lora_saver/lora_query_key_value\"  # PEFT微调模型的路径"
   ],
   "id": "618f34167708167b",
   "outputs": [],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-03T13:09:51.986495Z",
     "start_time": "2025-07-03T13:09:36.986867Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 在不计算梯度的情况下进行模型加载和预处理\n",
    "with torch.no_grad():\n",
    "    # 从预训练模型路径自动加载分词器\n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)\n",
    "    # 从预训练模型路径自动加载模型，并转换为半精度浮点数格式（节省显存），然后移动到GPU上\n",
    "    model = AutoModel.from_pretrained(model_dir, trust_remote_code=True).half().cuda()"
   ],
   "id": "8419777779015239",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/7 [00:00<?, ?it/s]"
      ],
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "ca493d2c287e47b2856c6906e5b5bd60"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-03T13:09:52.109932Z",
     "start_time": "2025-07-03T13:09:51.987745Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 使用PEFT微调模型对原始模型进行微调\n",
    "model = PeftModel.from_pretrained(model, peft_model_id)\n",
    "# 将模型设置为评估模式（关闭dropout、batchnorm等层的影响）\n",
    "model.eval()"
   ],
   "id": "6ab19a6e5531fd87",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PeftModelForCausalLM(\n",
       "  (base_model): LoraModel(\n",
       "    (model): ChatGLMForConditionalGeneration(\n",
       "      (transformer): ChatGLMModel(\n",
       "        (embedding): Embedding(\n",
       "          (word_embeddings): Embedding(65024, 4096)\n",
       "        )\n",
       "        (rotary_pos_emb): RotaryEmbedding()\n",
       "        (encoder): GLMTransformer(\n",
       "          (layers): ModuleList(\n",
       "            (0-27): 28 x GLMBlock(\n",
       "              (input_layernorm): RMSNorm()\n",
       "              (self_attention): SelfAttention(\n",
       "                (query_key_value): lora.Linear(\n",
       "                  (base_layer): Linear(in_features=4096, out_features=4608, bias=True)\n",
       "                  (lora_dropout): ModuleDict(\n",
       "                    (default): Dropout(p=0.05, inplace=False)\n",
       "                  )\n",
       "                  (lora_A): ModuleDict(\n",
       "                    (default): Linear(in_features=4096, out_features=8, bias=False)\n",
       "                  )\n",
       "                  (lora_B): ModuleDict(\n",
       "                    (default): Linear(in_features=8, out_features=4608, bias=False)\n",
       "                  )\n",
       "                  (lora_embedding_A): ParameterDict()\n",
       "                  (lora_embedding_B): ParameterDict()\n",
       "                )\n",
       "                (core_attention): CoreAttention(\n",
       "                  (attention_dropout): Dropout(p=0.0, inplace=False)\n",
       "                )\n",
       "                (dense): Linear(in_features=4096, out_features=4096, bias=False)\n",
       "              )\n",
       "              (post_attention_layernorm): RMSNorm()\n",
       "              (mlp): MLP(\n",
       "                (dense_h_to_4h): Linear(in_features=4096, out_features=27392, bias=False)\n",
       "                (dense_4h_to_h): Linear(in_features=13696, out_features=4096, bias=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (final_layernorm): RMSNorm()\n",
       "        )\n",
       "        (output_layer): Linear(in_features=4096, out_features=65024, bias=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-03T13:09:53.386592Z",
     "start_time": "2025-07-03T13:09:53.368776Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 定义对话历史和查询\n",
    "history = []  # 对话历史，初始为空列表\n",
    "query = \"你是谁\"  # 查询语句，即用户的输入问题\n",
    "role = \"user\"  # 角色，这里设置为\"user\"表示是用户的输入"
   ],
   "id": "bf3d9dd758c3378",
   "outputs": [],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-03T13:09:54.938039Z",
     "start_time": "2025-07-03T13:09:54.920867Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 使用分词器构建聊天输入\n",
    "inputs = tokenizer.build_chat_input(query, history=history, role=role)\n",
    "# 将输入数据移动到GPU上\n",
    "inputs = inputs.to('cuda')\n",
    "#inputs = inputs.to('cpu')\n",
    "print(inputs)"
   ],
   "id": "6cd200ff773daa76",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': tensor([[64790, 64792, 64795, 30910,    13, 30910, 34607, 55622, 64796]],\n",
      "       device='cuda:0'), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1]], device='cuda:0'), 'position_ids': tensor([[0, 1, 2, 3, 4, 5, 6, 7, 8]], device='cuda:0')}\n"
     ]
    }
   ],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-03T13:09:56.766395Z",
     "start_time": "2025-07-03T13:09:56.753122Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 定义结束标记和生成参数\n",
    "eos_token_id = [tokenizer.eos_token_id, tokenizer.get_command(\"<|user|>\"),\n",
    "                tokenizer.get_command(\"<|observation|>\")]  # 结束标记，包括普通的结束标记和特殊角色的结束标记\n",
    "gen_kwargs = {\"max_length\": 1200, \"num_beams\": 1, \"do_sample\": True, \"top_p\": 0.95,\n",
    "              \"temperature\": 0.95}  # 生成参数，包括最大长度、集束搜索宽度、是否进行采样、top-p参数和温度参数"
   ],
   "id": "9a0e130e21197576",
   "outputs": [],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-03T13:09:58.634717Z",
     "start_time": "2025-07-03T13:09:57.719176Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# 生成输出\n",
    "outputs = model.generate(**inputs, **gen_kwargs, eos_token_id=eos_token_id)\n",
    "# 对输出进行处理，去掉输入的部分并转换为列表形式\n",
    "outputs = outputs.tolist()[0][len(inputs[\"input_ids\"][0]):-1]"
   ],
   "id": "769200ccfda45f76",
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "probability tensor contains either `inf`, `nan` or element < 0",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mRuntimeError\u001B[0m                              Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[17], line 2\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# 生成输出\u001B[39;00m\n\u001B[1;32m----> 2\u001B[0m outputs \u001B[38;5;241m=\u001B[39m model\u001B[38;5;241m.\u001B[39mgenerate(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39minputs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mgen_kwargs, eos_token_id\u001B[38;5;241m=\u001B[39meos_token_id)\n\u001B[0;32m      3\u001B[0m \u001B[38;5;66;03m# 对输出进行处理，去掉输入的部分并转换为列表形式\u001B[39;00m\n\u001B[0;32m      4\u001B[0m outputs \u001B[38;5;241m=\u001B[39m outputs\u001B[38;5;241m.\u001B[39mtolist()[\u001B[38;5;241m0\u001B[39m][\u001B[38;5;28mlen\u001B[39m(inputs[\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124minput_ids\u001B[39m\u001B[38;5;124m\"\u001B[39m][\u001B[38;5;241m0\u001B[39m]):\u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m]\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\pytorch\\lib\\site-packages\\peft\\peft_model.py:1190\u001B[0m, in \u001B[0;36mPeftModelForCausalLM.generate\u001B[1;34m(self, *args, **kwargs)\u001B[0m\n\u001B[0;32m   1188\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_enable_peft_forward_hooks(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[0;32m   1189\u001B[0m         kwargs \u001B[38;5;241m=\u001B[39m {k: v \u001B[38;5;28;01mfor\u001B[39;00m k, v \u001B[38;5;129;01min\u001B[39;00m kwargs\u001B[38;5;241m.\u001B[39mitems() \u001B[38;5;28;01mif\u001B[39;00m k \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;129;01min\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mspecial_peft_forward_args}\n\u001B[1;32m-> 1190\u001B[0m         outputs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbase_model\u001B[38;5;241m.\u001B[39mgenerate(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n\u001B[0;32m   1191\u001B[0m \u001B[38;5;28;01melse\u001B[39;00m:\n\u001B[0;32m   1192\u001B[0m     outputs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mbase_model\u001B[38;5;241m.\u001B[39mgenerate(\u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\pytorch\\lib\\site-packages\\torch\\utils\\_contextlib.py:115\u001B[0m, in \u001B[0;36mcontext_decorator.<locals>.decorate_context\u001B[1;34m(*args, **kwargs)\u001B[0m\n\u001B[0;32m    112\u001B[0m \u001B[38;5;129m@functools\u001B[39m\u001B[38;5;241m.\u001B[39mwraps(func)\n\u001B[0;32m    113\u001B[0m \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21mdecorate_context\u001B[39m(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs):\n\u001B[0;32m    114\u001B[0m     \u001B[38;5;28;01mwith\u001B[39;00m ctx_factory():\n\u001B[1;32m--> 115\u001B[0m         \u001B[38;5;28;01mreturn\u001B[39;00m func(\u001B[38;5;241m*\u001B[39margs, \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mkwargs)\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\pytorch\\lib\\site-packages\\transformers\\generation\\utils.py:1525\u001B[0m, in \u001B[0;36mGenerationMixin.generate\u001B[1;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, assistant_model, streamer, negative_prompt_ids, negative_prompt_attention_mask, **kwargs)\u001B[0m\n\u001B[0;32m   1517\u001B[0m     input_ids, model_kwargs \u001B[38;5;241m=\u001B[39m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39m_expand_inputs_for_generation(\n\u001B[0;32m   1518\u001B[0m         input_ids\u001B[38;5;241m=\u001B[39minput_ids,\n\u001B[0;32m   1519\u001B[0m         expand_size\u001B[38;5;241m=\u001B[39mgeneration_config\u001B[38;5;241m.\u001B[39mnum_return_sequences,\n\u001B[0;32m   1520\u001B[0m         is_encoder_decoder\u001B[38;5;241m=\u001B[39m\u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39mconfig\u001B[38;5;241m.\u001B[39mis_encoder_decoder,\n\u001B[0;32m   1521\u001B[0m         \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mmodel_kwargs,\n\u001B[0;32m   1522\u001B[0m     )\n\u001B[0;32m   1524\u001B[0m     \u001B[38;5;66;03m# 13. run sample\u001B[39;00m\n\u001B[1;32m-> 1525\u001B[0m     \u001B[38;5;28;01mreturn\u001B[39;00m \u001B[38;5;28mself\u001B[39m\u001B[38;5;241m.\u001B[39msample(\n\u001B[0;32m   1526\u001B[0m         input_ids,\n\u001B[0;32m   1527\u001B[0m         logits_processor\u001B[38;5;241m=\u001B[39mprepared_logits_processor,\n\u001B[0;32m   1528\u001B[0m         logits_warper\u001B[38;5;241m=\u001B[39mlogits_warper,\n\u001B[0;32m   1529\u001B[0m         stopping_criteria\u001B[38;5;241m=\u001B[39mprepared_stopping_criteria,\n\u001B[0;32m   1530\u001B[0m         pad_token_id\u001B[38;5;241m=\u001B[39mgeneration_config\u001B[38;5;241m.\u001B[39mpad_token_id,\n\u001B[0;32m   1531\u001B[0m         eos_token_id\u001B[38;5;241m=\u001B[39mgeneration_config\u001B[38;5;241m.\u001B[39meos_token_id,\n\u001B[0;32m   1532\u001B[0m         output_scores\u001B[38;5;241m=\u001B[39mgeneration_config\u001B[38;5;241m.\u001B[39moutput_scores,\n\u001B[0;32m   1533\u001B[0m         return_dict_in_generate\u001B[38;5;241m=\u001B[39mgeneration_config\u001B[38;5;241m.\u001B[39mreturn_dict_in_generate,\n\u001B[0;32m   1534\u001B[0m         synced_gpus\u001B[38;5;241m=\u001B[39msynced_gpus,\n\u001B[0;32m   1535\u001B[0m         streamer\u001B[38;5;241m=\u001B[39mstreamer,\n\u001B[0;32m   1536\u001B[0m         \u001B[38;5;241m*\u001B[39m\u001B[38;5;241m*\u001B[39mmodel_kwargs,\n\u001B[0;32m   1537\u001B[0m     )\n\u001B[0;32m   1539\u001B[0m \u001B[38;5;28;01melif\u001B[39;00m generation_mode \u001B[38;5;241m==\u001B[39m GenerationMode\u001B[38;5;241m.\u001B[39mBEAM_SEARCH:\n\u001B[0;32m   1540\u001B[0m     \u001B[38;5;66;03m# 11. prepare beam search scorer\u001B[39;00m\n\u001B[0;32m   1541\u001B[0m     beam_scorer \u001B[38;5;241m=\u001B[39m BeamSearchScorer(\n\u001B[0;32m   1542\u001B[0m         batch_size\u001B[38;5;241m=\u001B[39mbatch_size,\n\u001B[0;32m   1543\u001B[0m         num_beams\u001B[38;5;241m=\u001B[39mgeneration_config\u001B[38;5;241m.\u001B[39mnum_beams,\n\u001B[1;32m   (...)\u001B[0m\n\u001B[0;32m   1548\u001B[0m         max_length\u001B[38;5;241m=\u001B[39mgeneration_config\u001B[38;5;241m.\u001B[39mmax_length,\n\u001B[0;32m   1549\u001B[0m     )\n",
      "File \u001B[1;32mE:\\ProgramData\\anaconda3\\envs\\pytorch\\lib\\site-packages\\transformers\\generation\\utils.py:2658\u001B[0m, in \u001B[0;36mGenerationMixin.sample\u001B[1;34m(self, input_ids, logits_processor, stopping_criteria, logits_warper, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, streamer, **model_kwargs)\u001B[0m\n\u001B[0;32m   2656\u001B[0m \u001B[38;5;66;03m# sample\u001B[39;00m\n\u001B[0;32m   2657\u001B[0m probs \u001B[38;5;241m=\u001B[39m nn\u001B[38;5;241m.\u001B[39mfunctional\u001B[38;5;241m.\u001B[39msoftmax(next_token_scores, dim\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m-\u001B[39m\u001B[38;5;241m1\u001B[39m)\n\u001B[1;32m-> 2658\u001B[0m next_tokens \u001B[38;5;241m=\u001B[39m \u001B[43mtorch\u001B[49m\u001B[38;5;241;43m.\u001B[39;49m\u001B[43mmultinomial\u001B[49m\u001B[43m(\u001B[49m\u001B[43mprobs\u001B[49m\u001B[43m,\u001B[49m\u001B[43m \u001B[49m\u001B[43mnum_samples\u001B[49m\u001B[38;5;241;43m=\u001B[39;49m\u001B[38;5;241;43m1\u001B[39;49m\u001B[43m)\u001B[49m\u001B[38;5;241m.\u001B[39msqueeze(\u001B[38;5;241m1\u001B[39m)\n\u001B[0;32m   2660\u001B[0m \u001B[38;5;66;03m# finished sentences should have their next token be a padding token\u001B[39;00m\n\u001B[0;32m   2661\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m eos_token_id \u001B[38;5;129;01mis\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28;01mNone\u001B[39;00m:\n",
      "\u001B[1;31mRuntimeError\u001B[0m: probability tensor contains either `inf`, `nan` or element < 0"
     ]
    }
   ],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-07-01T14:21:27.089558Z",
     "start_time": "2025-07-01T14:21:27.070548Z"
    }
   },
   "cell_type": "code",
   "source": [
    "\n",
    "# 使用分词器解码输出\n",
    "response = tokenizer.decode(outputs)\n",
    "# 处理响应，包括去除一些特殊标记和更新对话历史\n",
    "response, history = model.process_response(response, history)\n",
    "\n",
    "# 打印响应\n",
    "print(response)"
   ],
   "id": "a05b5c61ae93a86e",
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'outputs' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mNameError\u001B[0m                                 Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[9], line 2\u001B[0m\n\u001B[0;32m      1\u001B[0m \u001B[38;5;66;03m# 使用分词器解码输出\u001B[39;00m\n\u001B[1;32m----> 2\u001B[0m response \u001B[38;5;241m=\u001B[39m tokenizer\u001B[38;5;241m.\u001B[39mdecode(\u001B[43moutputs\u001B[49m)\n\u001B[0;32m      3\u001B[0m \u001B[38;5;66;03m# 处理响应，包括去除一些特殊标记和更新对话历史\u001B[39;00m\n\u001B[0;32m      4\u001B[0m response, history \u001B[38;5;241m=\u001B[39m model\u001B[38;5;241m.\u001B[39mprocess_response(response, history)\n",
      "\u001B[1;31mNameError\u001B[0m: name 'outputs' is not defined"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {},
   "cell_type": "code",
   "outputs": [],
   "execution_count": null,
   "source": "",
   "id": "b9b9d81780424c3a"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
